package DBOpt;

import java.util.*;
import java.io.*;

import javax.xml.parsers.*;

import java.sql.*;
import java.text.SimpleDateFormat;


//import com.mysql.jdbc.*;

public class XML_Converter {

 //
 Connection con;
 DocumentBuilderFactory factory;
 
  SimpleDateFormat createAtDateFormat;
  SimpleDateFormat outputTimestampFormat;
  SimpleDateFormat outputDateFormat,outputTimeFormat;
  
  protected  XML_Converter(){
	//convert datetime
	// Format: Mon Sep 01 04:00:53 +0000 2008
	 createAtDateFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss Z yyyy",Locale.US);
	 outputTimestampFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss",Locale.US);
	 outputDateFormat = new SimpleDateFormat("yyyy-MM-dd",Locale.US);
	 outputTimeFormat = new SimpleDateFormat("HH:mm:ss",Locale.US);
	 
 }
	
 
  
  public void convertAllFile(String rootDirName)
  {
	  factory = DocumentBuilderFactory.newInstance();  
      File rootDir = new File(rootDirName);
      Calendar cal = Calendar.getInstance();
      FileWriter  finishedDirFile;
      
      if (!rootDir.isDirectory()) 
      {
          // Either dir does not exist or is not a directory
    	  System.out.println(rootDirName+"is not a valid directory!");
    	  return;
      } 
    
      
      else 
      {
    	  String[] dir_children = rootDir.list();
    	  Arrays.sort(dir_children);

          // Use the factory to create a builder
    	  
    	  DocumentBuilder builder;
    	  try
    	  {
             builder = factory.newDocumentBuilder();
    	  }
    	  catch( Exception e ) 
  	      {
  	         e.printStackTrace();
  	         return;
  	       }
          
    	  //read the last finished date in finished list
    	  int index = 0;
    	  try
    	  {
    		  FileInputStream in = new FileInputStream("./.convert_finished_list");
        	  BufferedReader br = new BufferedReader(new InputStreamReader(in));
        	  		
        	  String strLine = null, tmp;

        	  while ((tmp = br.readLine()) != null)
        	  {
        	     strLine = tmp;
        	  }
        	  String lastLine = strLine;
        	  System.out.println("Previous Time:"+lastLine);
        	  if (lastLine != null){
        		  for(;index<dir_children.length;index++)
            	  {
            		String date = dir_children[index];
            		if (date.compareTo(lastLine)>0) break;
            		
            	  }
            	  if (index == dir_children.length)
            	  {
            		System.out.println ("No new data in incoming dir!");
            		return;
            	  }  
          		
        	  }
	 
    	  }
    	  catch (FileNotFoundException e)
    	  {
    		  System.out.println ("No previous finished dir");
    	  }
    	  catch( Exception e ) 
  	      {
  	         e.printStackTrace();
  	         return;
  	       }
    	  
    	  try
    	  {
    	   
    	   finishedDirFile = new FileWriter ("./.convert_finished_list", true);
    	   System.out.println("creart finishfile");
    	  }
    	  catch( Exception e ) 
  	      {
  	         e.printStackTrace();
  	         return;
  	       }
    	  
    	  System.out.println ("*** Start processing from:"+dir_children[index]+" ****");
      	  
    	  for (; index<dir_children.length; index++) 
          {
              String subDirName = rootDirName+"/"+dir_children[index];
              File subDir = new File(subDirName);
              if (!subDir.isDirectory())
              {
            	  System.out.println(subDir+" is not a sub directory! ignore it!");  
              }
              else
              {
            	  String[] file_children = subDir.list();
            	  for (int j=0; j<file_children.length; j++) 
                  {
            		  
           		      String s= file_children[j];
            		  int surfix_index =  s.lastIndexOf("xml");
            		  if (surfix_index != file_children[j].length()-3 ) 
            	      {
            	    	  System.out.println("Non-XML File:"+file_children[j]);
            	    	
            	      }
            	      else
            	      {
            	    	  
            	    	  String filename = subDirName+"/"+file_children[j];
                          System.out.println("XML File:"+filename);
                          
                          stripNonValidXMLCharacters(filename);
                          
                                      }
                     }
              }
          
              //finished one sub-dir
              try
        	  {
        	  finishedDirFile.write(dir_children[index]+"\n");
        	  finishedDirFile.flush();
        	  System.out.println("finished sub-directory:"+dir_children[index]);
        	  }
        	  catch( Exception e ) 
      	      {
      	         e.printStackTrace();
      	       }
          }
    	  
        }
      try{
      finishedDirFile.close();}
      catch( Exception e ){
	         e.printStackTrace();}
    
  
  }

  private boolean stripNonValidXMLCharacters(String inputfile){
	  String line;
	  boolean find = false;  
     
	  try{
		  String out = new String(); // Used to hold the output.
	      char current; // Used to reference the current character.
		  FileReader fr = new FileReader(inputfile);
		  BufferedReader br = new BufferedReader(fr);
		   while(true)
	       {
	    	   line = br.readLine();
	    	   if (line == null) break;
	           if (line.length()==0) continue;
	           for (int i = 0; i < line.length(); i++) {
	               current = line.charAt(i); // NOTE: No IndexOutOfBoundsException caught here; it should not happen.
	               if ((current == 0x9) ||
	                   (current == 0xA) ||
	                   (current == 0xD) ||
	                   ((current >= 0x20) && (current <= 0xD7FF)) ||
	                   ((current >= 0xE000) && (current <= 0xFFFD)) ||
	                   ((current >= 0x10000) && (current <= 0x10FFFF)))
	               {
	            	   out= out+current;
	               }
	               else
	               {
	            	   System.out.println("Find a invalid char: "+current);
	            	   find = true;
	               }
	                   
	           }
	           out= out+"\n";
	        }
	       if (find)
	       {
	    	   String filename = inputfile.substring(inputfile.lastIndexOf('/')+1,inputfile.length());
		       String outfilename = "../timline_saved/"+ filename;
		       System.out.println(outfilename);
		       
		       FileWriter outFile = new FileWriter (outfilename);
	    	   outFile.write(out);
			   outFile.close();
	       }
	       br.close();
	       fr.close();
	       return find;
	  }
	  catch (Exception e){//Catch exception if any
	  System.err.println("Error: " + e.getMessage());
	    }
	  return find;
  }  

  public static void main(String args[]) {
   
	if (args.length < 1)
	{
		System.out.println("Usage: xmlconverter <Source file directoy> ");
		System.exit(0) ;
	}
	try{  
	String rootDir = args[0];
	XML_Converter insertapp = new XML_Converter();
	
    insertapp.convertAllFile(rootDir);
    }
      catch( Exception e ) 
      {
         e.printStackTrace();
       }
  }
  
}
